#!/usr/local/php/bin/php
<?php
$DEBUG = false;
$runlog = '/tmp/recover.log';

$live_data_root_dir    = '/data5/data/wbng/page_storage/wbng/v1';
$legacy_data_root_dir  = '/data6/data/wbng/page_storage/wbng/v1';

$recover_filelist = '/tmp/recover.filelist';
$recover_status   = '/tmp/recover.run';

$symbol_data_root_dir  = '/home/s/apps/wbng/page_storage/data/wbng/v1';


if ($DEBUG) {
	$file_log = STDERR;

	$dbname = 'test';
	$dbuser = 'search';
	$dbpassword = '';
}
else {
	$file_log = fopen($runlog, 'a');

	$dbname = 'storage_page_v1';
	$dbuser = 'search';
	$dbpassword = '';
}

///
/// STEP. build recovery file list
if (!file_exists($recover_filelist)) {
	buildRecoveryFileList();
}

///
/// STEP. recover by recovery filelist
recover();

if (!$DEBUG) {
	fclose($file_log);
}

exit(0);

///////////////////////////////////////////////////////////////////////

function	getRecoverStatus($file_legacy)
{
	global	$legacy_data_root_dir, $live_data_root_dir;
	global	$recover_filelist, $recover_status;

	if (!file_exists($recover_status)) {
		return	false;
	}

	$status = array('stage' => false);
	$fh_status = fopen($recover_status, 'r');
	while(!feof($fh_status)) {
		$line = fgets($fh_status);
		$line = trim($line);
		if (strlen($line) < 2) {
			continue;
		}
		$f = split("\t", $line);
		if (0 == strcmp($file_legacy, $f[0])) {
			$status['file_legacy']  = $f[0];
			$status['cid']          = $f[1];
			$status['fileno_old']   = $f[2];
			$status['fileno_new']   = $f[3];
			$status['file_recover'] = $f[4];
			$status['stage']        = $f[5];
		}
	}
	fclose($fh_status);

	return $status;
}

function	updateRecoverStatus($status)
{
	global	$legacy_data_root_dir, $live_data_root_dir;
	global	$recover_filelist, $recover_status;

	$status = sprintf("%s\t%s\t%s\t%s\t%s\t%s\n", 
		$status['file_legacy'],
		$status['cid'],
		$status['fileno_old'],
		$status['fileno_new'],
		$status['file_recover'],
		$status['stage']
	);

	$fh_status = fopen($recover_status, 'a');
	fwrite($fh_status, $status);
	fclose($fh_status);
}

function	recoverOneFile($file_legacy)
{
	global	$legacy_data_root_dir, $live_data_root_dir;
	global	$recover_filelist, $recover_status;
	global	$symbol_data_root_dir;

	info("* recovering {$file_legacy} ...");

	// check whether the file has been recovered.
	// <file_legacy> <CID> <old file no> <new file no> <file_recover> <status:BEGIN|COPY|RECOVER|END>
	$status = getRecoverStatus($file_legacy);
	if ((false === $status) || (false === $status['stage'])) {
		// stage begin
		if (false === $status) {
			$status = array();
		}

		$path = dirname($file_legacy);
		$cid = basename($path);
		$filename = basename($file_legacy, '.data');
		$f = split('_', $filename);
		$fileno_old = $f[2];

		$path_live = $live_data_root_dir . '/' . $cid;
		$fileno_live_max = getMaxFileNo($path_live);
		$fileno_new = strval($fileno_live_max + 2);
		$file_recover = sprintf('%s/%s/v1_%d_%s.data', $live_data_root_dir, $cid, intval($cid), $fileno_new);

		$status['file_legacy']  = $file_legacy;
		$status['cid']          = $cid;
		$status['fileno_old']   = $fileno_old;
		$status['fileno_new']   = $fileno_new;
		$status['file_recover'] = $file_recover;
		$status['stage']        = 'BEGIN';
		info("  recover {$file_legacy} stage BEGIN");
		updateRecoverStatus($status);
	}

	if ('BEGIN' == $status['stage']) {
		$status['stage']        = 'COPY';
		info("  recover {$file_legacy} stage COPY");
		updateRecoverStatus($status);
	}

	if ('COPY' == $status['stage']) {
		$file_legacy  = $status['file_legacy'];
		if (!file_exists($file_legacy)) {
			info("  WARN: legacy file {$file_legacy} does NOT exists.");
			return	true;
		}

		$file_recover = $status['file_recover'];
		if (file_exists($file_recover)) {
			info("  WARN: recovery file {$file_recover} already exists.");
			$cid = $status['cid'];

			$path_live = $live_data_root_dir . '/' . $cid;
			$fileno_live_max = getMaxFileNo($path_live);
			$fileno_new = strval($fileno_live_max + 2);
			$file_recover = sprintf('%s/%s/v1_%d_%s.data', $live_data_root_dir, $cid, intval($cid), $fileno_new);

			$status['fileno_new']   = $fileno_new;
			$status['file_recover'] = $file_recover;
			info("  WARN: recovery file {$file_recover} will be recovered to.");
		}

		$ok = copy($file_legacy, $file_recover);
		if (!$ok) {
			info("  ERROR: fail to copy {$file_legacy} to {$file_recover}.");
			return false;
		}

		$status['stage']        = 'RECOVER';
		info("  recover {$file_legacy} stage RECOVER");
		updateRecoverStatus($status);
	}

	if ('RECOVER' == $status['stage']) {
		$file_recover = $status['file_recover'];
		if (!file_exists($file_recover)) {
			info("  ERROR: recovery file {$file_recover} does NOT exists.");
			return false;
		}
		
		$fileno_old = $status['fileno_old'];
		$fileno_new = $status['fileno_new'];
		$cid = $status['cid'];
		$ok = recoverFromFile($file_recover, $cid, $fileno_old, $fileno_new);
		if (!$ok) {
			info("  ERROR: fail to recover from file {$file_recover}.");
			return false;
		}

		$status['stage']        = 'END';
		info("  recover {$file_legacy} stage END");
		updateRecoverStatus($status);
	}

	if ('END' == $status['stage']) {
		$file_recover = $status['file_recover'];
		$file_name = basename($file_recover);
		$cid = $status['cid'];
		$file_link = sprintf("%s/%05d/%s", $symbol_data_root_dir, intval($cid), $file_name);
		$ok = symlink($file_recover, $file_link);
		if (!$ok) {
			info("  symlink ln -s {$file_recover} {$file_link} FAIL.");
		}
		else {
			info("  symlink ln -s {$file_recover} {$file_link} DONE.");
		}
		info("  recover {$file_legacy} stage DONE");
	}

	return	true;
}


function	recover()
{
	global	$legacy_data_root_dir, $live_data_root_dir;
	global	$recover_filelist, $recover_status;

	if (!file_exists($recover_filelist)) {
		info("No recovery filelist {$recover_filelist}, stop recovering.");
		return	false;
	}

	$fh_recovery = fopen($recover_filelist, 'r');
	if (!$fh_recovery) {
		info("Fail to open recovery filelist {$recover_filelist}.");
		return false;
	}

	while(!feof($fh_recovery)) {
		$line = fgets($fh_recovery);
		$line = trim($line);
		if (strlen($line) < 2) {
			continue;
		}

		$ok = recoverOneFile($line);
		if (!$ok) {
			info("  ERROR: fail to recover {$line}, stop.");
			return	false;
		}
	}

	fclose($fh_recovery);
	return	true;
}

function	buildRecoveryFileList()
{
	global	$legacy_data_root_dir, $live_data_root_dir;
	global	$recover_filelist;

	$filelist = array();

	$legacy_data_dir = glob($legacy_data_root_dir . '/*');
	foreach ($legacy_data_dir as $dir) {
		$cid_str = basename($dir);
		$cid_int = intval($cid_str);
		//echo 'CID:',$cid_str,'[',$cid_int,"]\n";

		$files = glob($dir . '/*');
		foreach ($files as $file) {
			info("* handling $file ...");
			$filename = basename($file);
			$file_live = sprintf("%s/%s/%s", $live_data_root_dir, $cid_str, $filename);
			if (!file_exists($file_live)) {
				info("  {$file_live} does NOT exists, skip {$file}");
				continue;
			}

			$no_change = false;
			$size_legacy = filesize($file);
			$size_live = filesize($file_live);
			if ($size_legacy == $size_live) {
				info("  md5ing {$file} ...");
				$md5_legacy = md5_file($file);
				info("  md5ing {$file_live} ...");
				$md5_live   = md5_file($file_live);
				if ($md5_legacy === $md5_live) {
					$no_change = true;
				}
			}
			if ($no_change) {
				info("  {$file} size:{$size_legacy} md5:{$md5_legacy} skipped");
				info("  {$file_live} size:{$size_live} md5:{$md5_live} skipped");
				continue;
			}

			$filelist[] = $file;
		}
	}

	$fh = fopen($recover_filelist, 'w');
	fwrite($fh, join("\n", $filelist));
	fclose($fh);
}

///////////////////////////////////////////////////////////////////////

function	recoverFromFile($filename, $cid, $fileno_old, $fileno_new)
{
	global	$dbname, $dbuser, $dbpassword;

	$table_name = sprintf('v1_idx_%d', intval($cid));
	$table = new DBTable($table_name, $dbname, $dbuser, $dbpassword);
	$ok = $table->open();
	if ( !$ok ) {
		info("  ERROR: fail to open table '$table'.");
		return false;
	}

	$fh = fopen($filename, 'r');
	if (!$fh) {
		info("  ERROR: fail to open recovery file {$filename}.");
		return false;
	}

	$count_records = 0;
	$count_bad_record = 0;
	$count_miss_index = 0;
	$count_dubious_length = 0;

	$count_recover = 0;
	$count_recover_fail = 0;
	$time_start = microtime(true);
	while(!feof($fh)) {
		$offset = ftell($fh);
		$line = fgets($fh);
		$line = trim($line, "\r\n");
		$length = strlen($line);
		if (strlen($line) < 10) {
			continue;
		}
		$count_records += 1;

		// parse XML, to get url
		$url = extractUrl($line);
		if (!$url) {
			info("  ERROR: cannot extract URL from record (offset=$offset, length=$length) in file {$filename}, skip it.");
			$count_bad_record += 1;
			continue;
		}

		// query DB to get the (fileno_idx, offset_idx, length_idx) by url
		list($fileno_idx, $offset_idx, $length_idx) = $table->query($url);
		if ((false === $fileno_idx) || (false === $offset_idx) || (false === $length_idx)) {
			info("  ERROR: index missed for url '{$url}'.");
			$count_miss_index += 1;
			continue;
		}
		if (!(intval($fileno_old) === intval($fileno_idx))) {
			//info("  DUBIOUS URL: {$url}");
			//info(sprintf("  FILE: fileno=%8d offset=%12d length=%8d", $fileno_old, $offset, $length));
			//info(sprintf("  IDX : fileno=%8d offset=%12d length=%8d", $fileno_idx, $offset_idx, $length_idx));
			//echo "FILENO {$fileno_old}<>{$fileno_idx}\n";
			continue;
		}
		if (!(intval($offset) === intval($offset_idx))) {
			//echo "OFFSET {$offset}<>{$offset_idx}\n";
			continue;
		}
		if (!(intval($length) === intval($length_idx))) {
			info("  DUBIOUS URL: {$url}");
			info(sprintf("  FILE: fileno=%8d offset=%12d length=%8d", $fileno_old, $offset, $length));
			info(sprintf("  IDX : fileno=%8d offset=%12d length=%8d", $fileno_idx, $offset_idx, $length_idx));

			$count_dubious_length += 1;
			continue;
		}

		$count_recover += 1;
		// update DB to set (fileno) if offset==offset_idx and length==length_idx by url and fileno_idx
		$ok = $table->updateFileno($url, $fileno_old, $fileno_new);
		if (!$ok) {
			info("  ERROR: UPDATE {$url} fileno {$fileno_old} --> {$fileno_new} failed.");
			$count_recover_fail += 1;
		}

		//break;
	}
	$time_stop = microtime(true);
	$time_last = $time_stop - $time_start;
	if (0 == $time_last) {
		$time_last = 0.001;
	}
	$tps_handle  = sprintf("%.2f", $count_records / $time_last);
	$tps_recover = sprintf("%.2f", $count_recover / $time_last);
	$time_last = sprintf("%.2f", $time_last);
	info("  STATIST records:{$count_records} bad_record:{$count_bad_record} miss_idx:{$count_miss_index} dubious_len:{$count_dubious_length} recover:{$count_recover} recover_fail:{$count_recover_fail} time_last:{$time_last} tps:{$tps_handle} tps_recover:{$tps_recover}");

	fclose($fh);
	$table->close();
	unset($table);
	return	true;
}

function	extractUrl($xml_record)
{
	$url = false;

	$dom = new DOMDocument();
	$dom->loadXML($xml_record);

	$xpath = new DOMXPath($dom);
	$url_path = '/document/key/url';
	$nodelist = $xpath->query($url_path);
	if ($nodelist->length < 1) {
		info("  ERROR: no '{$url_path}' xpath in xml data.");
		return	false;
	}
	$url_node = $nodelist->item(0);
	$url = $url_node->nodeValue;

	unset($url_node);
	unset($nodelist);
	unset($xpath);
	unset($dom);

	return trim($url);
}

class	DBTable
{
	private $dbh;
	private $dbname;
	private $dbuser;
	private $dbpassword;

	private	$table;
	private $sth_query;
	private $sth_update;

	public function __construct($table, $dbname='test', $dbuser='search', $dbpassword='')
	{
		$this->dbh = null;

		$this->dbname = $dbname;
		$this->dbuser = $dbuser;
		$this->dbpassword = $dbpassword;

		$this->table = $table;
		$this->sth_query = null;
		$this->sth_update = null;
	}

	public function open()
	{
		if ($this->isOpened()) {
			return	true;
		}

		try {
			$this->dbh = new PDO("mysql:host=localhost;dbname={$this->dbname}", $this->dbuser, $this->dbpassword);
		}
		catch (PDOException $ex) {
			info("  ERROR: fail to connect to dbname={$this->dbname} as '{$this->dbuser}' : {$ex->getMessage()}");
			return false;
		}

		$sql = "SELECT fileno,offset,length FROM {$this->table} WHERE url = :url";
		$this->sth_query = $this->dbh->prepare($sql);

		$sql = "UPDATE {$this->table} SET fileno = :fileno_new WHERE url = :url AND fileno = :fileno_old";
		$this->sth_update = $this->dbh->prepare($sql);
		return true;
	}

	public function isOpened()
	{
		return !empty($this->dbh);
	}

	public function close()
	{
		if ($this->isOpened()) {
			unset($this->dbh);
			$this->dbh = null;
		}
	}

	public function query_direct($url) 
	{
		$fileno = false;
		$offset = false;
		$length = false;

		$sql = "SELECT fileno,offset,length FROM {$this->table} WHERE url='{$url}'";
		$rowset = $this->dbh->query($sql);
		if (false === $rowset) {
			$error_code = $this->dbh->errorCode();
			$error_info = $this->dbh->errorInfo();
			info("  ERROR: SQL ({$sql}) return FALSE -- [$error_code] {$error_info[2]}.");
		}
		else {
			/*
			if ($rowset->rowCount() < 1) {
				$error_code = $rowset->errorCode();
				$error_info = $rowset->errorInfo();
				info("  ERROR: SQL ({$sql}) failure -- [$error_code] {$error_info[2]}");
			}
			*/
			foreach ($rowset as $row) {
				// this should be only ONE row
				$fileno = $row['fileno'];
				$offset = $row['offset'];
				$length = $row['length'];
			}
		}

		return array($fileno, $offset, $length);
	}

	public function query($url) 
	{
		$fileno = false;
		$offset = false;
		$length = false;

		$sql = "SELECT fileno,offset,length FROM {$this->table} WHERE url='{$url}'";
		$ok = $this->sth_query->execute(array('url' => $url));
		if (false === $ok) {
			$error_code = $this->dbh->errorCode();
			$error_info = $this->dbh->errorInfo();
			info("  ERROR: SQL ({$sql}) return FALSE -- [$error_code] {$error_info[2]}.");
		}
		else {
			$rowset = $this->sth_query->fetchAll();
			foreach ($rowset as $row) {
				// this should be only ONE row
				$fileno = $row['fileno'];
				$offset = $row['offset'];
				$length = $row['length'];
			}
		}

		return array($fileno, $offset, $length);
	}

	public function updateFileno($url, $fileno_old, $fileno_new)
	{
		$sql = "UPDATE {$this->table} SET fileno={$fileno_new} WHERE url='{$url}' AND fileno={$fileno_old}";
		$ok = $this->sth_update->execute(array('fileno_new' => $fileno_new, 'url' => $url, 'fileno_old' => $fileno_old));
		if (false === $ok) {
			$error_code = $this->dbh->errorCode();
			$error_info = $this->dbh->errorInfo();
			info("  ERROR: SQL ({$sql}) return FALSE -- [$error_code] {$error_info[2]}.");

			return	false;
		}
		else {
			$row_count = $this->sth_update->rowCount();
			if (1 != $row_count) {
				info("  ERROR: SQL ($sql) affected {$row_count} rows.");
				return false;
			}
		}

		return	true;
	}

}

///////////////////////////////////////////////////////////////////////

function	getMaxFileNo($path)
{
	$max_fileno = 0;

	$filenames = glob($path . '/*.data');
	foreach ($filenames as $filename) {
		$name = basename($filename, '.data');
		$f = split('_', $name);
		$fn = intval($f[2]);
		if ($fn > $max_fileno) {
			$max_fileno = $fn;
		}
	}

	return	$max_fileno;
}

function	info($msg)
{
	global	$file_log;

	fwrite($file_log, sprintf("%s ", date('Y-m-d H:i:s')));
	fwrite($file_log, $msg . "\n");
}
